{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from machine_lib import * "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"SUPER_ALPHA\",\"VISUALIZATION\",\"WORKDAY\"]}'\n"
     ]
    }
   ],
   "source": [
    "s = login()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = get_datafields(s, dataset_id = 'fundamental65', region='TWN', universe='TOP500')\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                     id                                        description                                            dataset                          category                                        subcategory region  delay universe    type  coverage  userCount  alphaCount  pyramidMultiplier themes\n",
      "417          fnd65_emgasia_sedol_d41isr  Relative Strength Index measures the average u...  {'id': 'fundamental65', 'name': 'Factor Ratios...  {'id': 'model', 'name': 'Model'}  {'id': 'model-estimates-models', 'name': 'Esti...    TWN      1   TOP500  MATRIX      1.00         20          78                1.6     []\n",
      "55       fnd65_allcap_sedol_chg3yepsast  It is defined as the trailing 12-month earning...  {'id': 'fundamental65', 'name': 'Factor Ratios...  {'id': 'model', 'name': 'Model'}  {'id': 'model-estimates-models', 'name': 'Esti...    TWN      1   TOP500  MATRIX      1.00         19          32                1.6     []\n",
      "0            fnd65_allcap_sedol_52whigh  It is defined as the month-end price divided b...  {'id': 'fundamental65', 'name': 'Factor Ratios...  {'id': 'model', 'name': 'Model'}  {'id': 'model-estimates-models', 'name': 'Esti...    TWN      1   TOP500  MATRIX      1.00         15          33                1.6     []\n",
      "296           fnd65_allcap_sedol_tstalp  It is defined as: (CORREL*(260-2)^.5)/(1-CORRE...  {'id': 'fundamental65', 'name': 'Factor Ratios...  {'id': 'model', 'name': 'Model'}  {'id': 'model-estimates-models', 'name': 'Esti...    TWN      1   TOP500  MATRIX      1.00         15          27                1.6     []\n",
      "269             fnd65_allcap_sedol_roic  It is defined as the trailing 12-month net inc...  {'id': 'fundamental65', 'name': 'Factor Ratios...  {'id': 'model', 'name': 'Model'}  {'id': 'model-estimates-models', 'name': 'Esti...    TWN      1   TOP500  MATRIX      1.00         14          20                1.6     []\n",
      "..                                  ...                                                ...                                                ...                               ...                                                ...    ...    ...      ...     ...       ...        ...         ...                ...    ...\n",
      "342  fnd65_emgasia_sedol_actrtn60m12lag  It is defined as the percent change in price f...  {'id': 'fundamental65', 'name': 'Factor Ratios...  {'id': 'model', 'name': 'Model'}  {'id': 'model-estimates-models', 'name': 'Esti...    TWN      1   TOP500  MATRIX      0.94          1           1                1.6     []\n",
      "97          fnd65_allcap_sedol_dtsm1_rd  It is defined as the standard deviation of a s...  {'id': 'fundamental65', 'name': 'Factor Ratios...  {'id': 'model', 'name': 'Model'}  {'id': 'model-estimates-models', 'name': 'Esti...    TWN      1   TOP500  MATRIX      1.00          1           1                1.6     []\n",
      "345              fnd65_emgasia_sedol_ad  It is defined as the most recently reported to...  {'id': 'fundamental65', 'name': 'Factor Ratios...  {'id': 'model', 'name': 'Model'}  {'id': 'model-estimates-models', 'name': 'Esti...    TWN      1   TOP500  MATRIX      1.00          1           1                1.6     []\n",
      "346          fnd65_emgasia_sedol_adpmoc  It is defined as the most recently reported su...  {'id': 'fundamental65', 'name': 'Factor Ratios...  {'id': 'model', 'name': 'Model'}  {'id': 'model-estimates-models', 'name': 'Esti...    TWN      1   TOP500  MATRIX      1.00          1           1                1.6     []\n",
      "663    fnd65_emgasia_sedol_yoychgroepct  It is defined as the yearly percentage change ...  {'id': 'fundamental65', 'name': 'Factor Ratios...  {'id': 'model', 'name': 'Model'}  {'id': 'model-estimates-models', 'name': 'Esti...    TWN      1   TOP500  MATRIX      1.00          1           1                1.6     []\n",
      "\n",
      "[406 rows x 14 columns]\n",
      "['winsorize(ts_backfill(fnd65_emgasia_sedol_d41isr, 120), std=4)', 'winsorize(ts_backfill(fnd65_allcap_sedol_chg3yepsast, 120), std=4)', 'winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4)']\n"
     ]
    }
   ],
   "source": [
    "selected_datafields_df = df[\n",
    "    (df[\"userCount\"] > 0)\n",
    "    ].sort_values(by=[\"userCount\"], ascending=False)\n",
    "print(selected_datafields_df)\n",
    "\n",
    "pc_fields = process_datafields(selected_datafields_df, \"matrix\")\n",
    "print(pc_fields[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['winsorize(ts_backfill(fnd65_emgasia_sedol_d41isr, 120), std=4)', 'ts_rank(winsorize(ts_backfill(fnd65_emgasia_sedol_d41isr, 120), std=4), 5)', 'ts_rank(winsorize(ts_backfill(fnd65_emgasia_sedol_d41isr, 120), std=4), 22)', 'ts_rank(winsorize(ts_backfill(fnd65_emgasia_sedol_d41isr, 120), std=4), 66)', 'ts_rank(winsorize(ts_backfill(fnd65_emgasia_sedol_d41isr, 120), std=4), 120)', 'ts_rank(winsorize(ts_backfill(fnd65_emgasia_sedol_d41isr, 120), std=4), 240)', 'ts_zscore(winsorize(ts_backfill(fnd65_emgasia_sedol_d41isr, 120), std=4), 5)', 'ts_zscore(winsorize(ts_backfill(fnd65_emgasia_sedol_d41isr, 120), std=4), 22)', 'ts_zscore(winsorize(ts_backfill(fnd65_emgasia_sedol_d41isr, 120), std=4), 66)', 'ts_zscore(winsorize(ts_backfill(fnd65_emgasia_sedol_d41isr, 120), std=4), 120)']\n",
      "34916\n"
     ]
    }
   ],
   "source": [
    "first_order = first_order_factory(pc_fields, ts_ops)\n",
    "print(first_order[:10])\n",
    "print(len(first_order))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "57104\n",
      "[('winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4)', 4), ('ts_rank(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_rank(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_rank(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_rank(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4)]\n"
     ]
    }
   ],
   "source": [
    "# Pad initial decay with alpha\n",
    "init_decay = 4\n",
    "fo_alpha_list = []\n",
    "for alpha in first_order:\n",
    "    fo_alpha_list.append((alpha, init_decay))\n",
    "print(len(fo_alpha_list))\n",
    "print(fo_alpha_list[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[('winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4)', 4), ('ts_rank(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_rank(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_rank(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_rank(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4), ('ts_rank(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_zscore(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_zscore(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_zscore(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_zscore(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4)], [('ts_zscore(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_delta(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_delta(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_delta(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_delta(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4), ('ts_delta(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_sum(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_sum(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_sum(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_sum(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4)], [('ts_sum(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_product(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_product(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_product(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_product(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4), ('ts_product(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_ir(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_ir(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_ir(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_ir(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4)], [('ts_ir(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_std_dev(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_std_dev(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_std_dev(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_std_dev(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4), ('ts_std_dev(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_mean(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_mean(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_mean(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_mean(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4)], [('ts_mean(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_arg_min(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_arg_min(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_arg_min(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_arg_min(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4), ('ts_arg_min(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_arg_max(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_arg_max(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_arg_max(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_arg_max(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4)], [('ts_arg_max(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_min_diff(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_min_diff(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_min_diff(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_min_diff(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4), ('ts_min_diff(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_max_diff(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_max_diff(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_max_diff(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_max_diff(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4)], [('ts_max_diff(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_returns(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_returns(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_returns(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_returns(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4), ('ts_returns(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_scale(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_scale(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_scale(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_scale(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4)], [('ts_scale(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_skewness(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_skewness(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_skewness(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_skewness(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4), ('ts_skewness(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_kurtosis(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_kurtosis(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_kurtosis(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_kurtosis(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4)], [('ts_kurtosis(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('ts_quantile(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 5)', 4), ('ts_quantile(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 22)', 4), ('ts_quantile(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 66)', 4), ('ts_quantile(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 120)', 4), ('ts_quantile(winsorize(ts_backfill(fnd65_allcap_sedol_52whigh, 120), std=4), 240)', 4), ('winsorize(ts_backfill(fnd65_allcap_sedol_60dsigma, 120), std=4)', 4), ('ts_rank(winsorize(ts_backfill(fnd65_allcap_sedol_60dsigma, 120), std=4), 5)', 4), ('ts_rank(winsorize(ts_backfill(fnd65_allcap_sedol_60dsigma, 120), std=4), 22)', 4), ('ts_rank(winsorize(ts_backfill(fnd65_allcap_sedol_60dsigma, 120), std=4), 66)', 4)], [('ts_rank(winsorize(ts_backfill(fnd65_allcap_sedol_60dsigma, 120), std=4), 120)', 4), ('ts_rank(winsorize(ts_backfill(fnd65_allcap_sedol_60dsigma, 120), std=4), 240)', 4), ('ts_zscore(winsorize(ts_backfill(fnd65_allcap_sedol_60dsigma, 120), std=4), 5)', 4), ('ts_zscore(winsorize(ts_backfill(fnd65_allcap_sedol_60dsigma, 120), std=4), 22)', 4), ('ts_zscore(winsorize(ts_backfill(fnd65_allcap_sedol_60dsigma, 120), std=4), 66)', 4), ('ts_zscore(winsorize(ts_backfill(fnd65_allcap_sedol_60dsigma, 120), std=4), 120)', 4), ('ts_zscore(winsorize(ts_backfill(fnd65_allcap_sedol_60dsigma, 120), std=4), 240)', 4), ('ts_delta(winsorize(ts_backfill(fnd65_allcap_sedol_60dsigma, 120), std=4), 5)', 4), ('ts_delta(winsorize(ts_backfill(fnd65_allcap_sedol_60dsigma, 120), std=4), 22)', 4), ('ts_delta(winsorize(ts_backfill(fnd65_allcap_sedol_60dsigma, 120), std=4), 66)', 4)]]\n",
      "572\n"
     ]
    }
   ],
   "source": [
    "# Load alphas to task pools\n",
    "pools = load_task_pool(fo_alpha_list, 10, 10)\n",
    "print(pools[0])\n",
    "print(len(pools))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "pool 23 task 9 post done\n",
      "pool 23 task 0 simulate done\n",
      "pool 23 task 1 simulate done\n",
      "pool 23 task 2 simulate done\n",
      "pool 23 task 3 simulate done\n",
      "pool 23 task 4 simulate done\n",
      "pool 23 task 5 simulate done\n",
      "pool 23 task 6 simulate done\n",
      "pool 23 task 7 simulate done\n",
      "pool 23 task 8 simulate done\n",
      "pool 23 task 9 simulate done\n",
      "pool 24 task 9 post done\n",
      "pool 24 task 0 simulate done\n",
      "pool 24 task 1 simulate done\n",
      "pool 24 task 2 simulate done\n",
      "pool 24 task 3 simulate done\n",
      "pool 24 task 4 simulate done\n",
      "pool 24 task 5 simulate done\n",
      "pool 24 task 6 simulate done\n",
      "pool 24 task 7 simulate done\n",
      "pool 24 task 8 simulate done\n",
      "pool 24 task 9 simulate done\n",
      "pool 25 task 9 post done\n",
      "pool 25 task 0 simulate done\n",
      "pool 25 task 1 simulate done\n",
      "pool 25 task 2 simulate done\n",
      "pool 25 task 3 simulate done\n",
      "pool 25 task 4 simulate done\n",
      "pool 25 task 5 simulate done\n",
      "pool 25 task 6 simulate done\n",
      "pool 25 task 7 simulate done\n",
      "pool 25 task 8 simulate done\n",
      "pool 25 task 9 simulate done\n",
      "pool 26 task 9 post done\n",
      "pool 26 task 0 simulate done\n",
      "pool 26 task 1 simulate done\n",
      "pool 26 task 2 simulate done\n",
      "pool 26 task 3 simulate done\n",
      "pool 26 task 4 simulate done\n",
      "pool 26 task 5 simulate done\n",
      "pool 26 task 6 simulate done\n",
      "pool 26 task 7 simulate done\n",
      "pool 26 task 8 simulate done\n",
      "pool 26 task 9 simulate done\n",
      "pool 27 task 9 post done\n",
      "pool 27 task 0 simulate done\n",
      "pool 27 task 1 simulate done\n",
      "pool 27 task 2 simulate done\n",
      "pool 27 task 3 simulate done\n",
      "pool 27 task 4 simulate done\n",
      "pool 27 task 5 simulate done\n",
      "pool 27 task 6 simulate done\n",
      "pool 27 task 7 simulate done\n",
      "pool 27 task 8 simulate done\n",
      "pool 27 task 9 simulate done\n",
      "pool 28 task 9 post done\n",
      "pool 28 task 0 simulate done\n",
      "pool 28 task 1 simulate done\n",
      "pool 28 task 2 simulate done\n",
      "pool 28 task 3 simulate done\n",
      "pool 28 task 4 simulate done\n",
      "pool 28 task 5 simulate done\n",
      "pool 28 task 6 simulate done\n",
      "pool 28 task 7 simulate done\n",
      "pool 28 task 8 simulate done\n",
      "pool 28 task 9 simulate done\n",
      "pool 29 task 9 post done\n",
      "pool 29 task 0 simulate done\n",
      "pool 29 task 1 simulate done\n",
      "pool 29 task 2 simulate done\n",
      "pool 29 task 3 simulate done\n",
      "pool 29 task 4 simulate done\n",
      "pool 29 task 5 simulate done\n",
      "pool 29 task 6 simulate done\n",
      "pool 29 task 7 simulate done\n",
      "pool 29 task 8 simulate done\n",
      "pool 29 task 9 simulate done\n",
      "pool 30 task 9 post done\n",
      "pool 30 task 0 simulate done\n",
      "pool 30 task 1 simulate done\n",
      "pool 30 task 2 simulate done\n",
      "pool 30 task 3 simulate done\n",
      "pool 30 task 4 simulate done\n",
      "pool 30 task 5 simulate done\n",
      "pool 30 task 6 simulate done\n",
      "pool 30 task 7 simulate done\n",
      "pool 30 task 8 simulate done\n",
      "pool 30 task 9 simulate done\n",
      "pool 31 task 9 post done\n",
      "pool 31 task 0 simulate done\n",
      "pool 31 task 1 simulate done\n",
      "pool 31 task 2 simulate done\n",
      "pool 31 task 3 simulate done\n",
      "pool 31 task 4 simulate done\n",
      "pool 31 task 5 simulate done\n",
      "pool 31 task 6 simulate done\n",
      "pool 31 task 7 simulate done\n",
      "pool 31 task 8 simulate done\n",
      "pool 31 task 9 simulate done\n",
      "pool 32 task 9 post done\n",
      "pool 32 task 0 simulate done\n",
      "pool 32 task 1 simulate done\n",
      "pool 32 task 2 simulate done\n",
      "pool 32 task 3 simulate done\n",
      "pool 32 task 4 simulate done\n",
      "pool 32 task 5 simulate done\n",
      "pool 32 task 6 simulate done\n",
      "pool 32 task 7 simulate done\n",
      "pool 32 task 8 simulate done\n",
      "pool 32 task 9 simulate done\n",
      "pool 33 task 9 post done\n",
      "pool 33 task 0 simulate done\n",
      "pool 33 task 1 simulate done\n",
      "pool 33 task 2 simulate done\n",
      "pool 33 task 3 simulate done\n",
      "pool 33 task 4 simulate done\n",
      "pool 33 task 5 simulate done\n",
      "pool 33 task 6 simulate done\n",
      "pool 33 task 7 simulate done\n",
      "pool 33 task 8 simulate done\n",
      "pool 33 task 9 simulate done\n",
      "pool 34 task 9 post done\n",
      "pool 34 task 0 simulate done\n",
      "pool 34 task 1 simulate done\n",
      "pool 34 task 2 simulate done\n",
      "pool 34 task 3 simulate done\n",
      "pool 34 task 4 simulate done\n",
      "pool 34 task 5 simulate done\n",
      "pool 34 task 6 simulate done\n",
      "pool 34 task 7 simulate done\n",
      "pool 34 task 8 simulate done\n",
      "pool 34 task 9 simulate done\n",
      "pool 35 task 9 post done\n",
      "pool 35 task 0 simulate done\n",
      "pool 35 task 1 simulate done\n",
      "pool 35 task 2 simulate done\n",
      "pool 35 task 3 simulate done\n",
      "pool 35 task 4 simulate done\n",
      "pool 35 task 5 simulate done\n",
      "pool 35 task 6 simulate done\n",
      "pool 35 task 7 simulate done\n",
      "pool 35 task 8 simulate done\n",
      "pool 35 task 9 simulate done\n",
      "pool 36 task 9 post done\n",
      "pool 36 task 0 simulate done\n",
      "pool 36 task 1 simulate done\n",
      "pool 36 task 2 simulate done\n",
      "pool 36 task 3 simulate done\n",
      "pool 36 task 4 simulate done\n",
      "pool 36 task 5 simulate done\n",
      "pool 36 task 6 simulate done\n",
      "pool 36 task 7 simulate done\n",
      "pool 36 task 8 simulate done\n",
      "pool 36 task 9 simulate done\n",
      "pool 37 task 9 post done\n",
      "pool 37 task 0 simulate done\n",
      "pool 37 task 1 simulate done\n",
      "pool 37 task 2 simulate done\n",
      "pool 37 task 3 simulate done\n",
      "pool 37 task 4 simulate done\n",
      "pool 37 task 5 simulate done\n",
      "pool 37 task 6 simulate done\n",
      "pool 37 task 7 simulate done\n",
      "pool 37 task 8 simulate done\n",
      "pool 37 task 9 simulate done\n",
      "pool 38 task 9 post done\n",
      "pool 38 task 0 simulate done\n",
      "pool 38 task 1 simulate done\n",
      "pool 38 task 2 simulate done\n",
      "pool 38 task 3 simulate done\n",
      "pool 38 task 4 simulate done\n",
      "pool 38 task 5 simulate done\n",
      "pool 38 task 6 simulate done\n",
      "pool 38 task 7 simulate done\n",
      "pool 38 task 8 simulate done\n",
      "pool 38 task 9 simulate done\n",
      "pool 39 task 9 post done\n",
      "pool 39 task 0 simulate done\n",
      "pool 39 task 1 simulate done\n",
      "pool 39 task 2 simulate done\n",
      "pool 39 task 3 simulate done\n",
      "pool 39 task 4 simulate done\n",
      "pool 39 task 5 simulate done\n",
      "pool 39 task 6 simulate done\n",
      "pool 39 task 7 simulate done\n",
      "pool 39 task 8 simulate done\n",
      "pool 39 task 9 simulate done\n",
      "pool 40 task 9 post done\n",
      "pool 40 task 0 simulate done\n",
      "pool 40 task 1 simulate done\n",
      "pool 40 task 2 simulate done\n",
      "pool 40 task 3 simulate done\n",
      "pool 40 task 4 simulate done\n",
      "pool 40 task 5 simulate done\n",
      "pool 40 task 6 simulate done\n",
      "pool 40 task 7 simulate done\n",
      "pool 40 task 8 simulate done\n",
      "pool 40 task 9 simulate done\n",
      "pool 41 task 9 post done\n",
      "pool 41 task 0 simulate done\n",
      "pool 41 task 1 simulate done\n",
      "pool 41 task 2 simulate done\n",
      "pool 41 task 3 simulate done\n",
      "pool 41 task 4 simulate done\n",
      "pool 41 task 5 simulate done\n",
      "pool 41 task 6 simulate done\n",
      "pool 41 task 7 simulate done\n",
      "pool 41 task 8 simulate done\n",
      "pool 41 task 9 simulate done\n",
      "pool 42 task 9 post done\n",
      "pool 42 task 0 simulate done\n",
      "pool 42 task 1 simulate done\n",
      "pool 42 task 2 simulate done\n",
      "pool 42 task 3 simulate done\n",
      "pool 42 task 4 simulate done\n",
      "pool 42 task 5 simulate done\n",
      "pool 42 task 6 simulate done\n",
      "pool 42 task 7 simulate done\n",
      "pool 42 task 8 simulate done\n",
      "pool 42 task 9 simulate done\n",
      "pool 43 task 9 post done\n",
      "pool 43 task 0 simulate done\n",
      "pool 43 task 1 simulate done\n",
      "pool 43 task 2 simulate done\n",
      "pool 43 task 3 simulate done\n",
      "pool 43 task 4 simulate done\n",
      "pool 43 task 5 simulate done\n",
      "pool 43 task 6 simulate done\n",
      "pool 43 task 7 simulate done\n",
      "pool 43 task 8 simulate done\n",
      "pool 43 task 9 simulate done\n",
      "pool 44 task 9 post done\n",
      "pool 44 task 0 simulate done\n",
      "pool 44 task 1 simulate done\n",
      "pool 44 task 2 simulate done\n",
      "pool 44 task 3 simulate done\n",
      "pool 44 task 4 simulate done\n",
      "pool 44 task 5 simulate done\n",
      "pool 44 task 6 simulate done\n",
      "pool 44 task 7 simulate done\n",
      "pool 44 task 8 simulate done\n",
      "pool 44 task 9 simulate done\n",
      "pool 45 task 9 post done\n",
      "pool 45 task 0 simulate done\n",
      "pool 45 task 1 simulate done\n",
      "pool 45 task 2 simulate done\n",
      "pool 45 task 3 simulate done\n",
      "pool 45 task 4 simulate done\n",
      "pool 45 task 5 simulate done\n",
      "pool 45 task 6 simulate done\n",
      "pool 45 task 7 simulate done\n",
      "pool 45 task 8 simulate done\n",
      "pool 45 task 9 simulate done\n",
      "pool 46 task 9 post done\n",
      "pool 46 task 0 simulate done\n",
      "pool 46 task 1 simulate done\n",
      "pool 46 task 2 simulate done\n",
      "pool 46 task 3 simulate done\n",
      "pool 46 task 4 simulate done\n",
      "pool 46 task 5 simulate done\n",
      "pool 46 task 6 simulate done\n",
      "pool 46 task 7 simulate done\n",
      "pool 46 task 8 simulate done\n",
      "pool 46 task 9 simulate done\n",
      "pool 47 task 9 post done\n",
      "pool 47 task 0 simulate done\n",
      "pool 47 task 1 simulate done\n",
      "pool 47 task 2 simulate done\n",
      "pool 47 task 3 simulate done\n",
      "pool 47 task 4 simulate done\n",
      "pool 47 task 5 simulate done\n",
      "pool 47 task 6 simulate done\n",
      "pool 47 task 7 simulate done\n",
      "pool 47 task 8 simulate done\n",
      "pool 47 task 9 simulate done\n",
      "pool 48 task 9 post done\n",
      "Not complete : https://api.worldquantbrain.com/simulations/4tWdOM54l4z2cishfFRgf2\n",
      "other\n",
      "pool 48 task 0 simulate done\n",
      "Not complete : https://api.worldquantbrain.com/simulations/sV5Sw5Is53b90uIgXFxfg\n",
      "other\n",
      "pool 48 task 1 simulate done\n",
      "Not complete : https://api.worldquantbrain.com/simulations/VFJTFa855bY9C7kDDXGkFe\n",
      "other\n",
      "pool 48 task 2 simulate done\n",
      "Not complete : https://api.worldquantbrain.com/simulations/1TInO7OE4zSbXb1il3Z13\n",
      "other\n",
      "pool 48 task 3 simulate done\n",
      "Not complete : https://api.worldquantbrain.com/simulations/21muYN1t84yybBvpnaFDDfe\n",
      "other\n",
      "pool 48 task 4 simulate done\n",
      "Not complete : https://api.worldquantbrain.com/simulations/2EExwy3jJ4YzccakDbbBV6x\n",
      "other\n",
      "pool 48 task 5 simulate done\n",
      "Not complete : https://api.worldquantbrain.com/simulations/1IQaRz1gv4EFbMmhWGM85xc\n",
      "other\n",
      "pool 48 task 6 simulate done\n",
      "Not complete : https://api.worldquantbrain.com/simulations/Xb72v4AO4WOc34154HVd3rR\n",
      "other\n",
      "pool 48 task 7 simulate done\n",
      "Not complete : https://api.worldquantbrain.com/simulations/1RPxL977f4v4aoP1dXuPsmL3\n",
      "other\n",
      "pool 48 task 8 simulate done\n",
      "Not complete : https://api.worldquantbrain.com/simulations/V7TYL33D4tdaUBjVf6QiAy\n",
      "other\n",
      "pool 48 task 9 simulate done\n",
      " loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      " loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      " loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      " loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      " loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      " loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      " loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      " loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      " loc key error\n",
      "b'{\"user\":{\"id\":\"JW80368\"},\"token\":{\"expiry\":14400.0},\"permissions\":[\"CONSULTANT\",\"MULTI_SIMULATION\",\"PROD_ALPHAS\",\"REFERRAL\",\"VISUALIZATION\",\"WORKDAY\"]}'\n",
      "pool 49 task 9 post done\n",
      "pool 49 task 0 simulate done\n",
      "pool 50 task 9 post done\n",
      "pool 50 task 0 simulate done\n",
      "pool 50 task 1 simulate done\n",
      "pool 50 task 2 simulate done\n",
      "pool 50 task 3 simulate done\n",
      "pool 50 task 4 simulate done\n",
      "pool 50 task 5 simulate done\n",
      "pool 50 task 6 simulate done\n",
      "pool 50 task 7 simulate done\n",
      "pool 50 task 8 simulate done\n",
      "pool 50 task 9 simulate done\n",
      "pool 51 task 9 post done\n",
      "pool 51 task 0 simulate done\n",
      "pool 51 task 1 simulate done\n",
      "pool 51 task 2 simulate done\n",
      "pool 51 task 3 simulate done\n",
      "pool 51 task 4 simulate done\n",
      "pool 51 task 5 simulate done\n",
      "pool 51 task 6 simulate done\n",
      "pool 51 task 7 simulate done\n",
      "pool 51 task 8 simulate done\n",
      "pool 51 task 9 simulate done\n",
      "pool 52 task 9 post done\n",
      "pool 52 task 0 simulate done\n",
      "pool 52 task 1 simulate done\n",
      "pool 52 task 2 simulate done\n",
      "pool 52 task 3 simulate done\n",
      "pool 52 task 4 simulate done\n",
      "pool 52 task 5 simulate done\n",
      "pool 52 task 6 simulate done\n",
      "pool 52 task 7 simulate done\n",
      "pool 52 task 8 simulate done\n",
      "pool 52 task 9 simulate done\n",
      "pool 53 task 9 post done\n",
      "pool 53 task 0 simulate done\n",
      "pool 53 task 1 simulate done\n",
      "pool 53 task 2 simulate done\n",
      "pool 53 task 3 simulate done\n",
      "pool 53 task 4 simulate done\n",
      "pool 53 task 5 simulate done\n",
      "pool 53 task 6 simulate done\n",
      "pool 53 task 7 simulate done\n",
      "pool 53 task 8 simulate done\n",
      "pool 53 task 9 simulate done\n",
      "pool 54 task 9 post done\n",
      "pool 54 task 0 simulate done\n",
      "pool 54 task 1 simulate done\n",
      "pool 54 task 2 simulate done\n",
      "pool 54 task 3 simulate done\n",
      "pool 54 task 4 simulate done\n",
      "pool 54 task 5 simulate done\n",
      "pool 54 task 6 simulate done\n",
      "pool 54 task 7 simulate done\n",
      "pool 54 task 8 simulate done\n",
      "pool 54 task 9 simulate done\n",
      "pool 55 task 9 post done\n",
      "pool 55 task 0 simulate done\n",
      "pool 55 task 1 simulate done\n",
      "pool 55 task 2 simulate done\n",
      "pool 55 task 3 simulate done\n",
      "pool 55 task 4 simulate done\n",
      "pool 55 task 5 simulate done\n",
      "pool 55 task 6 simulate done\n",
      "pool 55 task 7 simulate done\n",
      "pool 55 task 8 simulate done\n",
      "other\n",
      "pool 55 task 9 simulate done\n",
      "pool 56 task 9 post done\n"
     ]
    }
   ],
   "source": [
    "# Simulate First Order\n",
    "multi_simulate(pools, \"SUBINDUSTRY\", \"TWN\", \"TOP500\", 56, \"twn_fo_fund65_sub\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = get_datafields(s, dataset_id='news85', region='USA', universe='TOP3000', delay=1)\n",
    "print(df[df['type'] == \"VECTOR\"][\"id\"].tolist())\n",
    "print(process_datafields(df, \"vector\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
